package com.semanticweb.project.search.analyzer;

import java.io.Reader;

import org.apache.lucene.analysis.ASCIIFoldingFilter;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.LowerCaseFilter;
import org.apache.lucene.analysis.StopFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.standard.StandardTokenizer;
import org.apache.lucene.util.Version;

import com.semanticweb.project.search.analyzer.filters.CyrilicToLatinFilter;

public class SerbianAnalyzer extends Analyzer {

	public static final String[] STOP_WORDS = { "i", "a", "ili", "ali", "pa",
			"te", "da", "u", "po", "na" };

	@Override
	public TokenStream tokenStream(String fieldName, Reader reader) {
		System.out.println("Tokenizing field: " + fieldName);
		TokenStream result = new StandardTokenizer(Version.LUCENE_34, reader);
		result = new CyrilicToLatinFilter(result);
		result = new LowerCaseFilter(Version.LUCENE_34, result);
		result = new StopFilter(Version.LUCENE_34, result,
				StopFilter.makeStopSet(Version.LUCENE_34, STOP_WORDS));
		result = new ASCIIFoldingFilter(result);
		return result;
	}
}